S3: Default to signature=v4 when using an AWS endpoint

* S3: Default to signature=v4 when using an AWS endpoint, since some
  AWS regions need v4 and all support it. When host= is used to specify
  a different S3 host, the default remains signature=v2.
* webapp: Support setting up S3 buckets in regions that need v4
  signatures.

For the webapp, went ahead and added all current S3 regions
(except govcloud, which  is not usable by everyone).

Sponsored-by: Dartmouth College's DANDI project

diff --git a/CHANGELOG b/CHANGELOG
index 7c210b3bb9bf41c8f6585ec509d55408508bd676..434d2c160c93507e1b5682470feddfbd09fba75a 100644 (file)
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -5,6 +5,11 @@ git-annex (10.20250722) UNRELEASED; urgency=medium
   * Added git-remote-p2p-annex, which allows git pull and push to
     P2P networks provided by commands git-annex-p2p-<netname>
   * stack.yaml: Update to lts-24.2.
+  * S3: Default to signature=v4 when using an AWS endpoint, since some
+    AWS regions need v4 and all support it. When host= is used to specify
+    a different S3 host, the default remains signature=v2.
+  * webapp: Support setting up S3 buckets in regions that need v4
+    signatures.
 
  -- Joey Hess <id@joeyh.name>  Wed, 30 Jul 2025 13:45:42 -0400
 

diff --git a/Remote/Helper/AWS.hs b/Remote/Helper/AWS.hs
index 92608ee0a844691bf5c193499a0cf2c32b2a1808..2c9d535a85b58fc4fade17e63e684b3e7e0513ce 100644 (file)
--- a/Remote/Helper/AWS.hs
+++ b/Remote/Helper/AWS.hs
@@ -52,20 +52,40 @@ regionInfo service = map (\(t, r) -> (t, fromServiceRegion r)) $
        concatMap (\(t, l) -> map (t,) l) regions
   where
        regions =
-               [ ("US East (N. Virginia)", [S3Region "US", GlacierRegion "us-east-1"])
-               , ("US West (Oregon)", [BothRegion "us-west-2"])
+               -- Based on the list at https://docs.aws.amazon.com/general/latest/gr/s3.html
+               [ ("US East (Ohio)", [S3Region "us-east-2"])
+               , ("US East (N. Virginia)", [S3Region "US", GlacierRegion "us-east-1"])
                , ("US West (N. California)", [BothRegion "us-west-1"])
-               , ("EU (Ireland)", [S3Region "EU", GlacierRegion "eu-west-1"])
+               , ("US West (Oregon)", [BothRegion "us-west-2"])
+               , ("Africa (Cape Town)", [S3Region "af-south-1"])
+               , ("Asia Pacific (Hong Kong)", [S3Region "ap-east-1"])
+               , ("Asia Pacific (Hyderabad)", [S3Region "ap-south-2"])
+               , ("Asia Pacific (Jakarta)", [S3Region "ap-southeast-3"])
+               , ("Asia Pacific (Malaysia)", [S3Region "ap-southeast-5"])
+               , ("Asia Pacific (Melbourne)", [S3Region "ap-southeast-4"])
+               , ("Asia Pacific (Mumbai)", [S3Region "ap-south-1"])
+               , ("Asia Pacific (Osaka)", [S3Region "ap-northeast-3"])
+               , ("Asia Pacific (Seoul)", [S3Region "ap-northeast-2"])
                , ("Asia Pacific (Singapore)", [S3Region "ap-southeast-1"])
-               , ("Asia Pacific (Tokyo)", [BothRegion "ap-northeast-1"])
                , ("Asia Pacific (Sydney)", [S3Region "ap-southeast-2"])
+               , ("Asia Pacific (Taipei)", [S3Region "ap-east-2"])
+               , ("Asia Pacific (Thailand)", [S3Region "ap-southeast-7"])
+               , ("Asia Pacific (Tokyo)", [BothRegion "ap-northeast-1"])
+               , ("Canada (Central)", [S3Region "ca-central-1"])
+               , ("Canada West (Calgary)", [S3Region "ca-west-1"])
+               , ("EU (Frankfurt)", [BothRegion "eu-central-1"])
+               , ("EU (Ireland)", [S3Region "EU", GlacierRegion "eu-west-1"])
+               , ("Europe (London)", [S3Region "eu-west-2"])
+               , ("Europe (Milan)", [S3Region "eu-south-1"])
+               , ("Europe (Paris)", [S3Region "eu-west-3"])
+               , ("Europe (Spain)", [S3Region "eu-south-2"])
+               , ("Europe (Stockholm)", [S3Region "eu-north-1"])
+               , ("Europe (Zurich)", [S3Region "eu-central-2"])
+               , ("Israel (Tel Aviv)", [S3Region "il-central-1"])
+               , ("Mexico (Central)", [S3Region "mx-central-1"])
+               , ("Middle East (Bahrain)", [S3Region "me-south-1"])
+               , ("Middle East (UAE)", [S3Region "me-central-1"])
                , ("South America (São Paulo)", [S3Region "sa-east-1"])
-               -- These need signature V4 to be used, and currently v2 is
-               -- the default, so to add these would need other changes.
-               -- , ("EU (Frankfurt)", [BothRegion "eu-central-1"])
-               -- , ("Asia Pacific (Seoul)", [S3Region "ap-northeast-2"])
-               -- , ("Asia Pacific (Mumbai)", [S3Region "ap-south-1"])
-               -- , ("US East (Ohio)", [S3Region "us-east-2"])
                ]
 
        fromServiceRegion (BothRegion s) = s

diff --git a/Remote/S3.hs b/Remote/S3.hs
index b17b6c268ad6a3a3126ba8eaab55c2d63538c915..1fa8d44cd3e9eff60c588f25343d613dec452b98 100644 (file)
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -163,11 +163,12 @@ signatureField = Accepted "signature"
 
 data SignatureVersion 
        = SignatureVersion Int
+       | DefaultSignatureVersion
        | Anonymous
 
 signatureVersionParser :: RemoteConfigField -> FieldDesc -> RemoteConfigFieldParser
 signatureVersionParser f fd =
-       genParser go f (Just defver) fd
+       genParser go f (Just DefaultSignatureVersion) fd
                (Just (ValueDesc "v2 or v4 or anonymous"))
   where
        go "v2" = Just (SignatureVersion 2)
@@ -175,8 +176,6 @@ signatureVersionParser f fd =
        go "anonymous" = Just Anonymous
        go _ = Nothing
 
-       defver = SignatureVersion 2
-
 isAnonymous :: ParsedRemoteConfig -> Bool
 isAnonymous c = 
        case getRemoteConfigValue signatureField c of
@@ -984,16 +983,25 @@ s3Configuration _ua c = cfg
                Nothing
                        | port == 443 -> AWS.HTTPS
                        | otherwise -> AWS.HTTP
-       cfg = case getRemoteConfigValue signatureField c of
-               Just (SignatureVersion 4) -> 
-                       (S3.s3v4 proto endpoint False S3.SignWithEffort)
+       cfg = if usev4 $ getRemoteConfigValue signatureField c
+               then (S3.s3v4 proto endpoint False S3.SignWithEffort)
 #if MIN_VERSION_aws(0,24,0)
-                               { S3.s3Region = r }
+                       { S3.s3Region = r }
 #endif
-               _ -> (S3.s3 proto endpoint False)
+               else (S3.s3 proto endpoint False)
 #if MIN_VERSION_aws(0,24,0)
-                               { S3.s3Region = r }
+                       { S3.s3Region = r }
        
+       -- Use signature v4 for all AWS hosts by default, but don't use it by
+       -- default for other S3 hosts, which may not support it.
+       usev4 (Just DefaultSignatureVersion)
+               | h == AWS.s3DefaultHost = True
+               | otherwise = False
+       usev4 (Just (SignatureVersion 4)) = True
+       usev4 (Just (SignatureVersion _)) = False
+       usev4 (Just Anonymous) = False
+       usev4 Nothing = False
+
        r = encodeBS <$> getRemoteConfigValue regionField c
 #endif
 

diff --git a/doc/bugs/fails_to_authenticate_into_S3_for_initremote__63__/comment_4_d9a3fdfa34b780aafb5e5874fc8d98bc._comment b/doc/bugs/fails_to_authenticate_into_S3_for_initremote__63__/comment_4_d9a3fdfa34b780aafb5e5874fc8d98bc._comment
new file mode 100644 (file)
index 0000000..c75dc2e
--- /dev/null
+++ b/doc/bugs/fails_to_authenticate_into_S3_for_initremote__63__/comment_4_d9a3fdfa34b780aafb5e5874fc8d98bc._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2025-08-13T17:03:47Z"
+ content="""
+Made signature=v4 be used by default for AWS endpoints. Note that,
+if you want a special remote to be able to be used by an older version
+of git-annex, it would still make sense to explicitly specify signature=v4.
+"""]]

diff --git a/doc/special_remotes/S3.mdwn b/doc/special_remotes/S3.mdwn
index 3684432f2a605b650e4dd281abfffb4acb808fe6..96dfdb56d72bd501bc4e0938df496c5b4b648074 100644 (file)
--- a/doc/special_remotes/S3.mdwn
+++ b/doc/special_remotes/S3.mdwn
@@ -87,9 +87,8 @@ the S3 remote.
   indication that you need to use this.
 
 * `signature` - This controls the S3 signature version to use.
-  "v2" is currently the default, "v4" is needed to use some S3 services.
-  If you get some kind of authentication error, try "v4".
-  To access a S3 bucket anonymously, use "anonymous".
+  The default is "v4" when using Amazon S3, but "v2" when using other
+  hosts. To access a S3 bucket anonymously, use "anonymous".
 
 * `bucket` - S3 requires that buckets have a globally unique name, 
   so by default, a bucket name is chosen based on the remote name